package com.hex.ds.hdrs.watch.service;


import cn.hutool.core.date.DateField;
import cn.hutool.core.date.DatePattern;
import cn.hutool.core.date.DateUnit;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson.JSONObject;
import com.hex.bigdata.hbda.utils.FileUtil;
import com.hex.ds.hdrs.agent.po.*;
import com.hex.ds.hdrs.agent.service.AgentConfService;
import com.hex.ds.hdrs.agent.service.AgentHeartbeatLogService;
import com.hex.ds.hdrs.agent.service.AgentManager;
import com.hex.ds.hdrs.agent.service.AgentSession;
import com.hex.ds.hdrs.app.service.AppManager;
import com.hex.ds.hdrs.app.service.AppSession;
import com.hex.ds.hdrs.center.service.CenterManager;
import com.hex.ds.hdrs.common.constant.Const;
import com.hex.ds.hdrs.common.helper.RamHelper;
import com.hex.ds.hdrs.common.utils.HdrsDateUtil;
import com.hex.ds.hdrs.common.utils.ListUtil;
import com.hex.ds.hdrs.cron.po.CronTask;
import com.hex.ds.hdrs.cron.po.CronTaskDef;
import com.hex.ds.hdrs.cron.service.CronTaskDefService;
import com.hex.ds.hdrs.cron.service.CronTaskService;
import com.hex.ds.hdrs.engine.manager.RunningManager;
import com.hex.ds.hdrs.engine.manager.ToRunManager;
import com.hex.ds.hdrs.engine.service.QueueService;
import com.hex.ds.hdrs.log.po.RunningStateLog;
import com.hex.ds.hdrs.log.service.MailSendLogService;
import com.hex.ds.hdrs.log.service.RunningStateLogService;
import com.hex.ds.hdrs.period.job.po.JobTask;
import com.hex.ds.hdrs.period.job.po.PeriodJob;
import com.hex.ds.hdrs.period.job.service.PeriodJobService;
import com.hex.ds.hdrs.period.job.service.QueueJobInfService;
import com.hex.ds.hdrs.period.task.po.PeriodTask;
import com.hex.ds.hdrs.period.task.po.PeriodTaskConfig;
import com.hex.ds.hdrs.period.task.service.PeriodTaskConfigService;
import com.hex.ds.hdrs.period.task.service.PeriodTaskService;
import com.hex.ds.hdrs.period.task.util.FilterTaskHelper;
import com.hex.ds.hdrs.warn.po.WarnConf;
import com.hex.ds.hdrs.warn.po.WarnContent;
import com.hex.ds.hdrs.warn.service.WarnConfService;
import com.hex.ds.hdrs.warn.service.WarnContentService;
import com.hex.ds.hdrs.warn.service.WarnSendMethodService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.map.CaseInsensitiveMap;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Isolation;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;

import java.math.BigDecimal;
import java.math.RoundingMode;
import java.util.*;
import java.util.stream.Collectors;


@Slf4j
@Service
public class ScheduledService {

    @Autowired
    private AppManager appManager;
    @Autowired
    private AgentManager agentManager;
    @Autowired
    private CenterManager centerManager;
    @Autowired
    private ToRunManager toRunManager;
    @Autowired
    private RunningManager runningManager;
    @Autowired
    private WarnConfService warnConfService;
    @Autowired
    private AgentConfService agentConfService;
    @Autowired
    private RunningStateLogService runningStateLogService;
    @Autowired
    private AgentHeartbeatLogService agentHeartbeatLogService;
    @Autowired
    private PeriodTaskService periodTaskService;
    @Autowired
    private CronTaskService cronTaskService;
    @Autowired
    private QueueJobInfService queueJobInfService;
    @Autowired
    private PeriodJobService periodJobService;
    @Autowired
    private MailSendLogService mailSendLogService;
    @Autowired
    private WarnSendMethodService warnSendMethodService;
    @Autowired
    private WarnContentService warnContentService;
    @Autowired
    private CronTaskDefService cronTaskDefService;
    @Autowired
    private QueueService queueService;
    @Autowired
    private PeriodTaskConfigService periodTaskConfigService;

    @Value("${period.taskStep}")
    private String taskStep;

    @Value("${period.taskTimeOutFactor}")
    private Long taskTimeOutFactor;

    @Value("${period.killTimeOutTask}")
    private boolean killTimeOutTask;

    /**
     * 更新节点状态
     *
     * @param map
     */
    public void updateAgentStatus(Map map) {
        log.info("更新节点状态【ScheduledService.updateAgentStatus-开始】map: {}", map.toString());
        agentConfService.updateAgentStatus(map);
    }

    /**
     * @Method: updateFailTask
     * @Param: []
     * @Return: void
     * @Description： 查询失败的任务集合查询失败的任务集合
     * @Author： jd.chen
     * @Date： 2021/6/19 10:33
     * @Version： V1.0
     */
    @Transactional(propagation = Propagation.REQUIRED, rollbackFor = Exception.class)
    public void updateFailTask() {
        log.debug("【监控】查询失败的任务集合【ScheduledService.updateFailTask-开始】");
        try {
            Map<String, Object> qryMap = new HashMap<>(16);
            qryMap.put("taskStatus", Const.TASK_FAIL);
            List<PeriodTask> taskList = periodTaskService.qryPeriodTaskList(qryMap);
            if (ListUtil.isEmpty(taskList)) {
                return;
            }
            for (PeriodTask task : taskList) {
                updateFailTaskAndJob(task);
            }
        } catch (Exception e) {
            log.error("【updateRunningTask】异常: {}", e.getMessage(), e);
        }
    }

    /**
     * @Method: updateFailTaskAndJob <br>
     * @Param: [task] <br>
     * @Return: void <br>
     * @Description：更新失败任务<br>
     * @Author： gc.zuo<br>
     * @Date： 2021/12/8 10:15 <br>
     * @Version： V1.0<br>
     */
    @Transactional(propagation = Propagation.REQUIRED, rollbackFor = Exception.class)
    public void updateFailTaskAndJob(PeriodTask task) {
        //判断出错次数是否达到允许出错次数
        if (task.getErrCount() < task.getAllowErrCount()) {
            if (checkErrDifferDuration(task)){
                //更新任务和作业
                String upTs = HdrsDateUtil.getCurTs();
                this.periodJobService.upJobToToDo(task.getJobCode(), task.getBizDate(), upTs);
                Map<String, Object> map = new HashMap<>(16);
                map.put("upTs", upTs);
                map.put("taskCode", task.getTaskCode());
                map.put("bizDate", task.getBizDate());
                map.put("taskExecLog","任务失败重跑");
                periodTaskService.updateFailTask(map);
                log.debug("失败任务更新为待处理成功，taskCode:{},bizDate:{}", task.getTaskCode(), task.getBizDate());
            }
        }
    }

    public boolean checkErrDifferDuration(PeriodTask task){
        String endTs = StrUtil.isEmpty(task.getExecEndTs()) ? task.getUpTs() : task.getExecEndTs();
        // 将执行结束时间字符串转换为Date对象
        Date execEndDate = DateUtil.parse(endTs, "yyyy-MM-dd HH:mm:ss");
        // 将允许间隔时间字符串转换为秒数
        String errDifferDuration = StrUtil.isEmpty(task.getErrDifferDuration()) ? "0" : task.getErrDifferDuration();
        long errDifferDurationInSeconds = Long.parseLong(errDifferDuration);
        // 获取当前时间
        Date currentDate = new Date();
        // 计算当前时间与执行结束时间的时间差（秒）
        long differenceInSeconds = DateUtil.between(execEndDate, currentDate, cn.hutool.core.date.DateUnit.SECOND);
        // 判断时间差是否大于允许间隔时间
        return differenceInSeconds > errDifferDurationInSeconds;
    }


    /**
     * 停止执行任务
     */
    @Transactional(propagation = Propagation.REQUIRED, rollbackFor = Exception.class)
    public void killAgentTask() {
        killDBTask();
        killQueueTask();
        killOverTask();
    }

    /**
     * @Method killOverTask
     * @Param
     * @Return void
     * @Description 超过最大历史任务时间*超时因子，删除作业及任务
     * @Author Yin.Yang
     * @Date 2024/6/18 18:20
     * @Version V1.0
     */
    @Transactional(propagation = Propagation.REQUIRED, rollbackFor = Exception.class)
    public void killOverTask() {
        if (killTimeOutTask){
            try {
                List<PeriodTask> runningTaskList = findRunningTaskList();
                PeriodTaskConfig periodTaskConfig = periodTaskConfigService.queryOne();
                if (periodTaskConfig == null) {
                    log.info("【killOverTask】未找到任务配置，无法进行超时任务处理");
                    return;
                }
                long maxExecutionTime = Long.parseLong(periodTaskConfig.getMaxExecutionTime()) * taskTimeOutFactor;
                long currentTime = HdrsDateUtil.getMillis(new Date());
                for (PeriodTask periodTask : runningTaskList) {
                    try {
                        long execStartTime = HdrsDateUtil.strToLong(periodTask.getExecStTs(), HdrsDateUtil.YYYY_MM_DDHHMISS);
                        long execTime = (currentTime - execStartTime) / 1000;
                        if (execTime > maxExecutionTime) {
                            agentManager.killTask(periodTask);
                            periodJobService.delJobInfo(Collections.singletonList(periodTask.getJobPkId()));
                            cycleTaskRunningTimeOut(periodTask);
                        }
                    } catch (Exception e) {
                        log.error("【killOverTask】任务: {}超时未执行完成判断处理异常: {}", periodTask.getTaskCode(), e.getMessage(), e);
                    }
                }
            } catch (Exception e) {
                log.error("【killOverTask】异常: {}", e.getMessage(), e);
            }
        }
    }

    /**
     * @Method killQueueTask <br>
     * @Param  <br>
     * @Return void <br>
     * @Description 超时停止内存中正在执行任务<br>
     * @Author jian.qiao<br>
     * @Date 2024/1/5 11:36 <br>
     * @Version V1.0<br>
     */
    @Transactional(propagation = Propagation.REQUIRED, rollbackFor = Exception.class)
    public void killQueueTask(){
        try {
            //查询内存中正在执行的任务
            List<PeriodTask> runningTaskList = queueService.qryAllRunningTask();
            for (PeriodTask periodTask : runningTaskList) {
                try {
                    if (FilterTaskHelper.isOverTsTask(periodTask)) {
                        //任务超时未完成
                        PeriodTask task = periodTaskService.qryPeriodTaskInfo(periodTask.getTaskCode(), periodTask.getBizDate());
                        if (StrUtil.equalsAnyIgnoreCase(task.getTaskStatus(),Const.TASK_SUCCESS)) {
                            //判断db中该任务的状态是否为成功，成功则仅移除内存中数据（状态不一致）
                            runningManager.removeJob(periodTask, "任务超时未完成");
                            delTaskInSession(periodTask);
                            continue;
                        }
                        agentManager.killTask(periodTask);
                        updatePeriodTask(periodTask);
                        runningManager.removeJob(periodTask, "任务超时未完成");
                        delTaskInSession(periodTask);
                        cycleTaskRunningTimeOut(periodTask);
                    }
                } catch (Exception e) {
                    log.error("【killQueueTask】任务: {}超时未执行完成判断处理异常: {}", periodTask.getTaskCode(), e.getMessage(), e);
                }
            }
        } catch (Exception e){
            log.error("【killQueueTask】异常: {}", e.getMessage(), e);
        }
    }

    /**
     * @Method killDBTask <br>
     * @Param  <br>
     * @Return void <br>
     * @Description 超时停止DB中正在执行任务<br>
     * @Author jian.qiao<br>
     * @Date 2024/1/5 11:37 <br>
     * @Version V1.0<br>
     */
    @Transactional(propagation = Propagation.REQUIRED, rollbackFor = Exception.class)
    public void killDBTask(){
        try {
            List<PeriodTask> runningTaskList = findRunningTaskList();
            for (PeriodTask periodTask : runningTaskList) {
                try {
                    if (FilterTaskHelper.isOverTsTask(periodTask)) {
                        agentManager.killTask(periodTask);
                        updatePeriodTask(periodTask);
                        runningManager.removeJob(periodTask, "任务超时未完成");
                        delTaskInSession(periodTask);
                        cycleTaskRunningTimeOut(periodTask);
                    }
                } catch (Exception e) {
                    log.error("【killDBAgentTask】任务: {}超时未执行完成判断处理异常: {}", periodTask.getTaskCode(), e.getMessage(), e);
                }
            }
        } catch (Exception e) {
            log.error("【killDBAgentTask】异常: {}", e.getMessage(), e);
        }
    }

    /*
     * @Method: updatePeriodTask <br>
     * @Param: [task] <br>
     * @Return: void 超时任务更新成失败，并失败次数+1<br>
     * @Description：<br>
     * @Author： wz.li<br>
     * @Date： 2022/12/13 10:50 <br>
     * @Version： V2.0.2<br>
     */
    private void updatePeriodTask(PeriodTask task) {
        Map<String, Object> map = new HashMap<>(16);
        map.put("taskStatus", Const.TASK_FAIL);
        map.put("taskCode", task.getTaskCode());
        map.put("bizDate", task.getBizDate());
        map.put("upTs", HdrsDateUtil.getCurTs());
        map.put("execEndTs", HdrsDateUtil.getCurTs());
        periodTaskService.updateTask(map);
    }

    /*
     * @Method: findRunningTaskList <br>
     * @Param: [] <br>
     * @Return: java.util.List<com.hex.ds.hdrs.period.task.po.PeriodTask> <br>
     * @Description：获取所有正在执行的任务<br>
     * @Author： wz.li<br>
     * @Date： 2022/12/13 10:47 <br>
     * @Version： V2.0.2<br>
     */
    public List<PeriodTask> findRunningTaskList() {
        Map paramMap = new HashMap();
        paramMap.put("taskStatus", Const.TASK_RUNNING);
        return periodTaskService.qryPeriodTaskList(paramMap);
    }

    /**
     * 更新正在运行超时任务
     */
    public void updateRunningTask() {
        log.debug("【监控】更新未超时执行中的任务为待执行【ScheduledService.updateRunningTask-开始】");
        try {
            Map<String, Object> qryMap = new HashMap<>(16);
            qryMap.put("taskStatus", Const.TASK_RUNNING);
            List<PeriodTask> taskList = periodTaskService.qryPeriodTaskList(qryMap);

            Map<String, Object> map = new HashMap<>(16);
            for (PeriodTask task : taskList) {
                if (FilterTaskHelper.isOverTsTask(task)) {
                    if (runningManager.isContainsTask(task.getTaskCode())) {
                        continue;
                    } else {
                        map.put("taskStatus", Const.TASK_TODO_RUN);
                        map.put("taskCode", task.getTaskCode());
                        map.put("bizDate", task.getBizDate());
                        map.put("upTs", HdrsDateUtil.getCurTs());
                        map.put("execStTsIsNull", true);
                        map.put("execEndTsIsNull", true);
                        if (periodTaskService.updateTask(map) == 1) {
                            AppSession appSession = appManager.getValidAppSession(task.getAppCode());
                            appSession.delTaskSession(task);
                            AgentSession agentSession = agentManager.getAgentSession(task.getAgentCode());
                            agentSession.delTaskSession(task);
                        }
                    }
                }
            }
        } catch (Exception e) {
            log.error("【updateRunningTask】异常: {}", e.getMessage(), e);

        }
    }

    /**
     * 监控toRunList 和 map差异
     */
    public void toRunMemoryDeal() {
        try {
            List<JobTask> toRunJobTaskList = toRunManager.getToRunJobList();
            Map<String, JobTask> toRunJobMap = toRunManager.getToRunJobMap();
            RamHelper.delJobInListNotInMap(toRunJobTaskList, toRunJobMap);
        } catch (Exception e) {
            log.error("【toRunMemoryDeal】异常: {}", e.getMessage(), e);

        }
    }

    /**
     * 监控runningList 和 map差异
     */
    public void runningMemoryDeal() {
        try {
            List<JobTask> runningJobTaskList = runningManager.getRunJobList();
            Map<String, JobTask> runningJobMap = runningManager.getRunJobMap();
            RamHelper.delJobInListNotInMap(runningJobTaskList, runningJobMap);

            List<PeriodTask> runningTaskList = runningManager.getRunTaskList();
            Map<String, PeriodTask> runningTaskMap = runningManager.getRunTaskMap();
            List<PeriodTask> delTaskList = RamHelper.delTaskInListNotInMap(runningTaskList, runningTaskMap);

            for (PeriodTask task : delTaskList) {
                this.delTaskInSession(task);
            }
        } catch (Exception e) {
            log.error("【toRunMemoryDeal】异常: {}", e.getMessage(), e);

        }
    }

    /**
     * 删除内存session中的任务
     *
     * @param task
     */
    public void delTaskInSession(PeriodTask task) {
        AppSession appSession = appManager.getAppSession(task.getAppCode());
        if (appSession.containTask(task)) {
            appSession.delTaskSession(task);
        }

        AgentSession agentSession = agentManager.getAgentSession(task.getAgentCode());
        if (agentSession.containTask(task)) {
            agentSession.delTaskSession(task);
        }
    }

    /*
     * @Method: agentOfflineNotice <br>
     * @Param: [agentConf] <br>
     * @Return: void <br>
     * @Description：调度服务掉线告警<br>
     * @Author： wz.li<br>
     * @Date： 2023/1/18 13:57 <br>
     * @Version： V2.0.2<br>
     */
    public void agentOfflineNotice(AgentConf agentConf) {
        List<WarnConf> warnConfList = warnConfService.getOffAgentInfo(Const.AGENT_WARN_OFF_LINE, agentConf.getAgentCode());
        for (WarnConf warnConf : warnConfList) {
            Map noticInf = warnConfService.getNoticInf(warnConf);
            StringBuffer sb = new StringBuffer();
            sb.append("尊敬的管理员，您好！").append("\r\n");
            sb.append("   Agent：" + agentConf.getAgentName() + "下线,下线时间: " + HdrsDateUtil.getCurrentTime());
            warnSendMethodService.add(noticInf, "Agent掉线告警", sb.toString(), null, warnConf.getPkId());
        }
    }

    /**
     * Agent 上/下线汇总
     */
    public void agentOnAndOffline(String nowTime) {
        log.debug("Agent 上/下线汇总【ScheduledService.agentOnAndOffline-开始】");
        try {
            List<WarnConf> warnConfs = this.warnConfService.selectWarnConfList(Const.AGENT_CLIENT_CHANGE);
            for (WarnConf warnConf : warnConfs) {
                if (nowTime.equals(warnConf.getParam1())) {
                    String prevTime = HdrsDateUtil.getCurDate() + " 00:00";
                    List<RunningStateLog> logList = this.getAgentOnAndOfflineLog(warnConf, nowTime, prevTime);
                    Map noticInf = this.warnConfService.getNoticInf(warnConf);
                    String mailContent = this.getAgentOnAndOfflineMailConent(nowTime, logList, prevTime);
                    this.warnSendMethodService.add(noticInf, null, mailContent, null, warnConf.getPkId());
                }
            }
        } catch (Exception e) {
            log.error("Agent 上/下线汇总异常: {}", e.getMessage(), e);
        }
        log.debug("Agent 上/下线汇总【ScheduledService.agentOnAndOffline-结束】");
    }

    private String getAgentOnAndOfflineMailConent(String nowTime, List<RunningStateLog> logList, String prevTime) {
        StringBuffer sb = new StringBuffer();

        sb.append("尊敬的管理员，您好！").append("\r\n");
        sb.append("    截止今日").append(nowTime).append(",Agent上下线情况如下:").append("\r\n");
        if (CollectionUtils.isNotEmpty(logList)) {
            for (RunningStateLog log : logList) {
                String type = (Const.AGENT_WARN_OFF_LINE_NAME.equals(log.getRemindTitle())) ? "掉线" : "上线";
                sb.append("    Agent: " + log.getSrcName() + ",在" + log.getCrtTs() + "时: " + type).append("\r\n");
            }
        } else {
            sb.append("    所有Agent都运行良好！");
        }
        return sb.toString();
    }

    private List<RunningStateLog> getAgentOnAndOfflineLog(WarnConf warnConf, String nowTime, String prevTime) {
        String endTime = HdrsDateUtil.format(new Date(), "yyyy-MM-dd") + " " + nowTime + ":00";
        List<AgentConf> agentList = this.getMoniterAgentList(warnConf);
        List<String> agentCodeList = agentList.stream().map(AgentConf::getAgentCode).collect(Collectors.toList());
        List<RunningStateLog> logList = this.runningStateLogService.getAgentOnAndOffline(prevTime, endTime, agentCodeList);
        return logList;
    }

    /**
     * Agent资源监控（CPU、内存、IO、磁盘容量）
     */
    public void agentResourceWarnNotic() {
        log.debug("Agent 资源监控【ScheduledService.AgentResourceWarnNotic-开始】");
        try {
            List<AgentConf> agentConfList = this.agentConfService.qryAgentConfList();
            List<WarnConf> cpuWarnConfList = this.warnConfService.selectWarnConfList(Const.AGENT_WARN_CPU);
            List<WarnConf> ramWarnConfList = this.warnConfService.selectWarnConfList(Const.AGENT_WARN_RAM);
            List<WarnConf> diskWarnConfList = this.warnConfService.selectWarnConfList(Const.AGENT_WARN_DISK);
            for (AgentConf agentConf : agentConfList) {
                if (Const.AGENT_ENABLE_STATUS.equals(agentConf.getAgentStatus()) && !Const.AGENT_HA_STATUS_STOP.equals(agentConf.getAgentHaStatus())) {
                    List<AgentHeartbeatLog> hbList = this.getAgentHBLog(agentConf.getAgentCode());
                    if (hbList.size() > 0) {
                        // CPU 判断
                        this.agentCpuWarnNotice(agentConf, cpuWarnConfList, hbList);
                        // 内存判断
                        this.agentRamWarnNotice(agentConf, ramWarnConfList, hbList);
                        // 磁盘容量判断
                        this.agentDiskWarnNotice(agentConf, diskWarnConfList, hbList);
                    }
                }
            }
        } catch (Exception e) {
            log.error("Agent 资源预警异常：{}", e.getMessage(), e);
        }
        log.debug("Agent 资源监控【ScheduledService.AgentResourceWarnNotic-结束】");
    }

    /**
     * 内存队列数量预警
     */
    public void queueNumMonitor() {
        List<WarnConf> warnConfList = this.warnConfService.selectWarnConfList(Const.RAM_QUEUE_WARN);
        if (CollectionUtils.isNotEmpty(warnConfList)) {
            String startTime = HdrsDateUtil.format(DateUtils.addMinutes(new Date(), -5), "yyyy-MM-dd HH:mm:ss");
            List<Map> dataList = this.queueJobInfService.getBlockAndDependmentNum4(Const.QUEUE_CENTER_APP_CODE, startTime);
            Integer todoQueueAvgNum = this.getQueueAvgNumByType(Const.QUEUE_INF_TO_DO, dataList);
            Integer toRunQueueAvgNum = this.getQueueAvgNumByType(Const.QUEUE_INF_TO_RUN, dataList);
            Integer runningQueueAvgNum = this.getQueueAvgNumByType(Const.QUEUE_INF_RUNNING, dataList);
            this.checkQueueNum(warnConfList, todoQueueAvgNum, toRunQueueAvgNum, runningQueueAvgNum);
        }
    }

    /**
     * 根据内存类型获取内存队列大小
     *
     * @param type
     * @param dataList
     * @return
     */
    private Integer getQueueAvgNumByType(String type, List<Map> dataList) {
        BigDecimal bigDecimal = BigDecimal.ZERO;
        Integer count = 0;
        for (Map dataMap : dataList) {
            dataMap = new CaseInsensitiveMap(dataMap);
            String queueType = (String) dataMap.get("type");
            Long num = new Double(String.valueOf(dataMap.get("num"))).longValue();
            if (type.equals(queueType)) {
                count++;
                bigDecimal = bigDecimal.add(new BigDecimal(num));
            }
        }
        return count == 0 ? 0 : bigDecimal.divide(new BigDecimal(count), 2, RoundingMode.DOWN).intValue();
    }

    /**
     * DB任务实例数量预警
     */
    public void dbTaskNumMonitor() {
        List<WarnConf> warnConfList = this.warnConfService.selectWarnConfList(Const.DB_TASK_WARN);
        if (CollectionUtils.isNotEmpty(warnConfList)) {
            log.debug("DB 任务实例数量过多预警配置：{}", warnConfList.size());
            Integer cycleTaskTotal = this.periodTaskService.countTask(new HashMap(16));
            log.debug("DB 周期任务实例数据总数为：{}", cycleTaskTotal);
            Integer cycleTaskTodo = this.periodTaskService.countTask(Const.TASK_TODO_RUN);
            log.debug("DB 周期任务未完成实例数据总数为：{}", cycleTaskTodo);
            Integer cronTaskTotal = this.cronTaskService.countTask(new HashMap(16));
            log.debug("DB 定时任务实例数据总数为：{}", cronTaskTotal);
            this.checkDbTaskCount(warnConfList, cycleTaskTotal, cycleTaskTodo, cronTaskTotal);
        }
    }


    /**
     * Agent Cpu预警
     */
    private void agentCpuWarnNotice(AgentConf agentConf, List<WarnConf> warnConfList, List<AgentHeartbeatLog> hbList) {
        try {
            for (WarnConf warnConf : warnConfList) {
                // 该配置是否监听该agent
                if (this.checkAgentWarn(agentConf, warnConf)) {
                    String result = this.checkAgentCpu(warnConf, hbList);
                    if (StringUtils.isNotBlank(result) && isSendAgentResources(warnConf)) {
                        StringBuffer sb = new StringBuffer();
                        sb.append("尊敬的管理员，您好！").append("\r\n");
                        sb.append("     Agent：" + agentConf.getAgentName() + "的CPU使用率过高，均值为:" + result);
                        this.doAddWarnLog(Const.TYPE_AGENT, agentConf.getAgentCode(), agentConf.getAgentName(), Const.WARN_LEVEL_WARN
                                , Const.AGENT_WARN_CPU_NAME, sb.toString(), warnConf);
                    }
                }
            }
        } catch (Exception e) {
            log.error("Agent: {} CPU预警处理异常：{}", agentConf.getAgentName(), e.getMessage(), e);
        }
    }

    /*
     * @Method: isSendAgentResources <br>
     * @Param: [warnConf] <br>
     * @Return: void <br>
     * @Description：判断资源超过阈值是否要发送预警（3个小时发送一次）<br>
     * @Author： wz.li<br>
     * @Date： 2023/1/18 14:42 <br>
     * @Version： V2.0.2<br>
     */
    private boolean isSendAgentResources(WarnConf warnConf) {
        WarnContent warnContent = warnContentService.queryByWarnConfId(warnConf.getPkId());
        return warnContent == null || cn.hutool.core.date.DateUtil.between(cn.hutool.core.date.DateUtil.parseDateTime(warnContent.getCrtTs()), new Date(), DateUnit.HOUR) >= 3;
    }

    /**
     * 检查告警配置中用户是否选择自定义agent
     *
     * @param agentConf
     * @param warnConf
     * @return
     */
    private boolean checkAgentWarn(AgentConf agentConf, WarnConf warnConf) {
        if (Const.WARN_IS_REGION_ALL.equals(warnConf.getIsRegionAll())) {
            return true;
        }
        if (StringUtils.isNotBlank(warnConf.getRegionJson())) {
            String[] agentCodes = warnConf.getRegionJson().split(",");
            if (Arrays.asList(agentCodes).contains(agentConf.getAgentCode())) {
                return true;
            }
        }
        return false;
    }

    /**
     * Agent 内存预警
     */
    private void agentRamWarnNotice(AgentConf agentConf, List<WarnConf> warnConfList, List<AgentHeartbeatLog> hbList) {
        try {
            for (WarnConf warnConf : warnConfList) {
                if (this.checkAgentWarn(agentConf, warnConf)) {
                    String result = this.checkAgentRam(warnConf, hbList);
                    if (StringUtils.isNotBlank(result) && isSendAgentResources(warnConf)) {
                        StringBuffer sb = new StringBuffer();
                        sb.append("尊敬的管理员，您好！").append("\r\n");
                        sb.append("     Agent：" + agentConf.getAgentName() + "的空余内存不足，" + result);
                        this.doAddWarnLog(Const.TYPE_AGENT, agentConf.getAgentCode(), agentConf.getAgentName(), Const.WARN_LEVEL_WARN
                                , Const.AGENT_WARN_RAM_NAME, sb.toString(), warnConf);
                    }
                }
            }
        } catch (Exception e) {
            log.error("Agent: {} 内存预警处理异常：{}", agentConf.getAgentName(), e.getMessage(), e);
        }
    }

    /**
     * Agent 磁盘容量预警
     */
    public void agentDiskWarnNotice(AgentConf agentConf, List<WarnConf> warnConfList, List<AgentHeartbeatLog> hbList) {
        try {
            for (WarnConf warnConf : warnConfList) {
                if (checkAgentWarn(agentConf, warnConf)) {
                    String result = this.checkAgentDisk(hbList, warnConf);
                    if (StringUtils.isNotBlank(result) && isSendAgentResources(warnConf)) {
                        StringBuffer sb = new StringBuffer();
                        sb.append("尊敬的管理员，您好！").append("\r\n");
                        sb.append("     Agent：" + agentConf.getAgentName() + "的磁盘空间不足，" + result);
                        this.doAddWarnLog(Const.TYPE_AGENT, agentConf.getAgentCode(), agentConf.getAgentName(), Const.WARN_LEVEL_WARN
                                , Const.AGENT_WARN_DISK_NAME, sb.toString(), warnConf);
                    }
                }
            }
        } catch (Exception e) {
            log.error("Agent: {} 内存预警处理异常：{}", agentConf.getAgentName(), e.getMessage(), e);
        }
    }

    /**
     * 向运行记录表中插入数据
     *
     * @param type
     * @param code
     * @param name
     * @param level
     * @param title
     * @param errorInf
     * @param warnConf
     */
    private void doAddWarnLog(String type, String code, String name, String level, String title, String errorInf,
                              WarnConf warnConf) {
        Map noticeMap = this.warnConfService.getNoticInf(warnConf);
        this.doAddWarnLog(type, code, name, level, title, errorInf, noticeMap, warnConf.getPkId());
    }

    /**
     * 向运行记录表中插入数据
     *
     * @param type
     * @param code
     * @param name
     * @param level
     * @param title
     * @param errorInf
     * @param noticeMap
     */
    private void doAddWarnLog(String type, String code, String name, String level, String title, String errorInf,
                              Map noticeMap, String confId) {
        if (this.runningStateLogService.checkNeedSaveLog(type, code, title, errorInf)) {
            List<String> mailList = (List<String>) noticeMap.get(Const.WARN_TYPE_MAIL);
            List<String> phoneList = (List<String>) noticeMap.get(Const.WARN_TYPE_MES);

            RunningStateLog log = new RunningStateLog();
            log.setSrcType(type);
            log.setSrcCode(code);
            log.setSrcName(name);
            log.setWarnLevel(level);
            log.setRemindTitle(title);
            log.setRemindContent(errorInf);
            log.setCrtTs(HdrsDateUtil.getCurrentTime());
            log.setIsMailNotic(CollectionUtils.isNotEmpty(mailList) ? "1" : "2");
            log.setIsSmsNotic(CollectionUtils.isNotEmpty(phoneList) ? "1" : "2");
            log.setMailAddress(StringUtils.join(mailList, ","));
            log.setPhones(StringUtils.join(phoneList, ","));
            log.setIsSend(this.runningStateLogService.checkNeedSend(log) ? Const.WARN_NOT_SEND : Const.WARN_NOT_NEED_SEND);
            this.runningStateLogService.insert(log);
            //添加到邮件发送日志
            //this.mailSendLogService.add(StringUtils.join(mailList, ","), title, errorInf, false);
        }
        this.warnSendMethodService.add(noticeMap, title, errorInf, null, confId);
    }

    /**
     * 1、获取要监控的agent List
     *
     * @param warnConfs
     * @return
     */
    private List<AgentConf> getMoniterAgentList(List<WarnConf> warnConfs) {
        log.debug("【ScheduledService.agentOffLineManager】");
        if (CollectionUtils.isEmpty(warnConfs)) {
            return new ArrayList<>();
        }
        List<String> agentCodeList = new ArrayList<>();
        for (WarnConf warnConf : warnConfs) {
            if ("1".equals(warnConf.getIsRegionAll())) {
                return this.agentConfService.qryAgentConfList();
            } else {
                agentCodeList.addAll(Arrays.asList(warnConf.getRegionJson().split(",")));
            }
        }
        return this.agentConfService.qryAgentConfByAppCodes(agentCodeList);
    }

    /**
     * 中DB中获取要监控的Agent list
     *
     * @param warnConf
     * @return
     */
    private List<AgentConf> getMoniterAgentList(WarnConf warnConf) {
        if ("1".equals(warnConf.getIsRegionAll())) {
            return this.agentConfService.qryAgentConfList();
        } else {
            List<String> agentCodeList = Arrays.asList(warnConf.getRegionJson().split(","));
            return this.agentConfService.qryAgentConfByAppCodes(agentCodeList);
        }
    }

    /**
     * 2.判断Agent是否掉线并获取掉线Agent集合
     *
     * @param agentConfList
     * @return
     */
    private List<AgentConf> getAgentOffLineList(List<AgentConf> agentConfList) {
        List<AgentConf> offlineAgentList = new ArrayList<>();
        for (AgentConf agentConf : agentConfList) {
            if (StringUtils.isBlank(agentConf.getLastReceveTime())) {
                offlineAgentList.add(agentConf);
            } else {
                long nowD = HdrsDateUtil.getMillis(new Date());
                long heartBeatTime = HdrsDateUtil.parser(agentConf.getLastReceveTime(), HdrsDateUtil.YYYY_MM_DDHHMISS).getTime();
                if ((nowD - heartBeatTime) > 2 * 60 * 1000) {
                    offlineAgentList.add(agentConf);
                }
            }
        }
        return offlineAgentList;
    }

    /**
     * 判断Agent的CPU是否异常
     * 条件最近5分钟内的心跳信息，最少3条数据比较 平均cpu是否大于阀值
     *
     * @param warnConf
     * @return
     */
    private String checkAgentCpu(WarnConf warnConf, List<AgentHeartbeatLog> hbList) {
        BigDecimal total = BigDecimal.ZERO;
        for (AgentHeartbeatLog log : hbList) {
            AgentCpu agentCpu = JSONObject.parseObject(log.getCpuInfo(), AgentCpu.class);
            total = total.add(BigDecimal.valueOf(agentCpu.getCpuCombined()));
        }
        BigDecimal avg = total.divide(new BigDecimal(hbList.size()), 2, RoundingMode.DOWN);
        if (avg.compareTo(new BigDecimal(warnConf.getParam1())) > 0) {
            return avg.toString();
        }
        return null;
    }

    /**
     * 判断Agent的磁盘容量是否异常
     *
     * @param warnConf
     * @return
     */
    private String checkAgentRam(WarnConf warnConf, List<AgentHeartbeatLog> hbList) {
        BigDecimal total = BigDecimal.ZERO;
        for (AgentHeartbeatLog log : hbList) {
            AgentMem agentMem = JSONObject.parseObject(log.getMemoryInfo(), AgentMem.class);
            if ("1".equals(warnConf.getParam6())) {
                total = total.add(BigDecimal.valueOf(agentMem.getMemUsedRate()));
            } else {
                total = total.add(BigDecimal.valueOf(agentMem.getMemFree() / 1024));
            }
        }
        BigDecimal avg = total.divide(new BigDecimal(hbList.size()), 2, RoundingMode.DOWN);
        int result = avg.compareTo(new BigDecimal(warnConf.getParam1()));
        if ("1".equals(warnConf.getParam6()) && result > 0) {
            BigDecimal freeRam = new BigDecimal(100).subtract(avg);
            return "该节点内存空闲内存为: " + freeRam + "%";
        }
        if ("2".equals(warnConf.getParam6()) && result < 0) {
            return "该节点内存空闲内存大小为: " + FileUtil.getSize(avg.longValue() * 1024 * 1024);
        }
        return null;
    }

    /**
     * 判断磁盘容量是否超出
     *
     * @param hbList
     * @param warnConf
     * @return
     */
    private String checkAgentDisk(List<AgentHeartbeatLog> hbList, WarnConf warnConf) {
        BigDecimal total = BigDecimal.ZERO;
        for (AgentHeartbeatLog log : hbList) {
            AgentFileSys agentFileSys = JSONObject.parseObject(log.getFileSysInfo(), AgentFileSys.class);
            if ("1".equals(warnConf.getParam6())) {
                total = total.add(BigDecimal.valueOf(agentFileSys.getFileUsedRate()));
            } else {
                total = total.add(BigDecimal.valueOf(agentFileSys.getFileFree() / 1024));

            }
        }
        BigDecimal avg = total.divide(new BigDecimal(hbList.size()), 2, RoundingMode.DOWN);
        int result = avg.compareTo(new BigDecimal(warnConf.getParam1()));
        if ("1".equals(warnConf.getParam6()) && result > 0) {
            BigDecimal freeRam = new BigDecimal(100).subtract(avg);
            return "磁盘空间剩余: " + freeRam + "%";
        }
        if ("2".equals(warnConf.getParam6()) && result < 0) {
            return "磁盘空间空余大小为: " + FileUtil.getSize(avg.longValue() * 1024 * 1024);
        }
        return null;
    }


    /**
     * 获取某个Agent最近5分钟的心跳信息
     *
     * @param agentCode
     * @return
     */
    private List<AgentHeartbeatLog> getAgentHBLog(String agentCode) {
        String startTime = HdrsDateUtil.format(DateUtils.addMinutes(new Date(), -5), "yyyy-MM-dd HH:mm:ss");
        Map paramMap = new HashMap(16);
        paramMap.put("agentCode", agentCode);
        paramMap.put("startTime", startTime);
        List<AgentHeartbeatLog> hbList = this.agentHeartbeatLogService.qryHbLog(paramMap);
        return hbList;
    }

    /**
     * 判断DB任务实例是否达到预警值
     *
     * @param warnConfList
     * @param cycleTaskTotal
     * @param cycleTaskTodo
     * @param cronTaskTotal
     * @return
     */
    private void checkDbTaskCount(List<WarnConf> warnConfList, Integer cycleTaskTotal, Integer
            cycleTaskTodo, Integer cronTaskTotal) {
        for (WarnConf warnConf : warnConfList) {
            Integer cycleTaskTotalThreshold = Integer.valueOf(warnConf.getParam1());
            Integer cycleTaskTodoThreshold = Integer.valueOf(warnConf.getParam2());
            Integer cronTaskTotalThreshold = Integer.valueOf(warnConf.getParam3());

            StringBuffer sb = new StringBuffer();
            if (cycleTaskTotalThreshold > 0 && cycleTaskTotal > cycleTaskTotalThreshold) {
                sb.append("    ").append("周期任务实例总数：" + cycleTaskTotal + "    预警阀值：" + cycleTaskTotalThreshold).append("\r\n");
            }
            if (cycleTaskTodoThreshold > 0 && cycleTaskTodo > cycleTaskTodoThreshold) {
                sb.append("    ").append("周期任务未执行任务数：" + cycleTaskTodo + "    预警阀值：" + cycleTaskTodoThreshold).append("\r\n");
            }
            if (cronTaskTotalThreshold > 0 && cronTaskTotal > cronTaskTotalThreshold) {
                sb.append("    ").append("定时任务实例总数：" + cronTaskTotal + "    预警阀值：" + cronTaskTotalThreshold).append("\r\n");
            }
            WarnContent warnContent = warnContentService.queryByWarnConfId(warnConf.getPkId());
            if (sb.length() > 0 && (warnContent == null
                    || cn.hutool.core.date.DateUtil.between(cn.hutool.core.date.DateUtil.parseDateTime(warnContent.getCrtTs()), new Date(), DateUnit.DAY) >= 1)) {
                StringBuffer sb2 = new StringBuffer();
                sb2.append("尊敬的管理员，您好！").append("\r\n");
                sb2.append(sb);
                this.doAddWarnLog(Const.TYPE_CENTER, centerManager.getCenter().getCenterCode(), centerManager.getCenter().getCenterName(),
                        Const.WARN_LEVEL_WARN, "数据库任务实例过多", sb2.toString(), warnConf);
            }
        }
    }

    /**
     * 判断内存队列数量是否超过预警值
     *
     * @param warnConfList
     * @param todoQueueAvgNum
     * @param toRunQueueAvgNum
     * @param runningQueueAvgNum
     */
    private void checkQueueNum(List<WarnConf> warnConfList, Integer todoQueueAvgNum, Integer
            toRunQueueAvgNum, Integer runningQueueAvgNum) {
        for (WarnConf warnConf : warnConfList) {
            Integer todoQueueThreshold = Integer.valueOf(warnConf.getParam1());
            Integer toRunQueueThreshold = Integer.valueOf(warnConf.getParam2());
            Integer runningQueueThreshold = Integer.valueOf(warnConf.getParam3());

            StringBuffer sb = new StringBuffer();
            if (todoQueueThreshold > 0 && todoQueueAvgNum > todoQueueThreshold) {
                sb.append("     待执行队列作业总数：" + todoQueueAvgNum + ",预警阀值：" + todoQueueThreshold).append("\r\n");
            }
            if (toRunQueueThreshold > 0 && toRunQueueAvgNum > toRunQueueThreshold) {
                sb.append("     可执行队列作业总数：" + toRunQueueAvgNum + ",预警阀值：" + toRunQueueThreshold).append("\r\n");
            }
            if (runningQueueThreshold > 0 && runningQueueAvgNum > runningQueueThreshold) {
                sb.append("     正在执行队列作业总数：" + runningQueueAvgNum + ",预警阀值：" + runningQueueThreshold).append("\r\n");
            }
            WarnContent warnContent = warnContentService.queryByWarnConfId(warnConf.getPkId());
            if (sb.length() > 0 && (warnContent == null
                    || cn.hutool.core.date.DateUtil.between(cn.hutool.core.date.DateUtil.parseDateTime(warnContent.getCrtTs()), new Date(), DateUnit.HOUR) >= 1)) {
                StringBuffer sb2 = new StringBuffer();
                sb2.append("尊敬的管理员，您好！").append("\r\n");
                sb2.append(sb);
                this.doAddWarnLog(Const.TYPE_CENTER, centerManager.getCenter().getCenterCode(), centerManager.getCenter().getCenterName(),
                        Const.WARN_LEVEL_WARN, "内存队列数据太多", sb2.toString(), warnConf);
            }
        }
    }

    /**
     * 定期汇总任务执行情况
     */
    public void taskReportTimingSummary(String nowTime) {
        log.debug("定期汇总任务执行情况【ScheduledService.taskReportTimingSummary-开始】");
        try {
            List<WarnConf> warnConfList = this.warnConfService.selectWarnConfList(Const.TASK_TIMING_SUMMARY, false);
            for (WarnConf warnConf : warnConfList) {
                // 当前时间点是否是预警时间点、app是否活跃
                if (HdrsDateUtil.checkEqualTimePoints(warnConf.getParam1(), nowTime)
                        && appManager.getValidAppSession(warnConf.getAppCode()) != null) {
                    String appName = appManager.getValidAppSession(warnConf.getAppCode()).getAppName();
                    Map noticInf = this.warnConfService.getNoticInf(warnConf);
                    String subject = appName + "任务执行情况汇总";
                    String mailContent = this.getTaskTimingSummaryMailContent(warnConf, nowTime, appName);
                    this.warnSendMethodService.add(noticInf, subject, mailContent, null, warnConf.getPkId());
                }
            }
        } catch (Exception e) {
            log.error("定期汇总任务执行情况异常：{}", e.getMessage());
        }
        log.debug("定期汇总任务执行情况【ScheduledService.taskReportTimingSummary-结束】");
    }

    private String getTaskTimingSummaryMailContent(WarnConf warnConf, String time, String appName) {
        String bizDate = HdrsDateUtil.format(new Date(), "yyyyMMdd");
        StringBuffer sb = new StringBuffer();
        sb.append("尊敬的管理员，您好！").append(Const.MAIL_LINE_FEED);
        sb.append("    截止").append(bizDate + " " + time).append(",应用:" + appName + "的作业及任务执行情况如下:").append("\n\n");

        List<Map> periodJobList = this.periodJobService.countGroupByStatus(warnConf.getAppCode());
        sb.append("总的周期作业执行情况:").append(Const.MAIL_LINE_FEED);
        sb.append("   完成的作业数量:").append(RamHelper.getCycleJobStatusCount(periodJobList, Const.JOB_DO_SUCCESS));
        sb.append("   未完成的作业数量:").append(RamHelper.getCycleJobStatusCount(periodJobList, Const.JOB_TO_DO));
        sb.append("\n\n\n");

        List<Map> currentDayPeriodJobMap = this.periodJobService.countGroupByStatus(warnConf.getAppCode(), bizDate);
        sb.append("今日业务的周期作业执行情况:").append(Const.MAIL_LINE_FEED);
        sb.append("   完成的作业数量:").append(RamHelper.getCycleJobStatusCount(currentDayPeriodJobMap, Const.JOB_DO_SUCCESS));
        sb.append("   未完成的作业数量:").append(RamHelper.getCycleJobStatusCount(currentDayPeriodJobMap, Const.JOB_TO_DO));
        sb.append("\n\n\n");

        List<Map> periodTaskList = this.periodTaskService.getStatusCountByAppCode(warnConf.getAppCode());
        sb.append("总的周期任务执行情况:").append(Const.MAIL_LINE_FEED);
        sb.append("   待执行任务数量:").append(RamHelper.getCycleStatusCount(periodTaskList, Const.TASK_TODO_RUN));
        sb.append(",正在执行任务数量:").append(RamHelper.getCycleStatusCount(periodTaskList, Const.TASK_RUNNING)).append(Const.MAIL_LINE_FEED);
        sb.append("   执行成功任务数量:").append(RamHelper.getCycleStatusCount(periodTaskList, Const.TASK_SUCCESS));
        sb.append(",执行失败任务数量:").append(RamHelper.getCycleStatusCount(periodTaskList, Const.TASK_FAIL));
        sb.append("   置虚任务数量:").append(RamHelper.getCycleStatusCount(periodTaskList, Const.TASK_HAND_SUCCESS));
        sb.append("   暂停中任务数量:").append(RamHelper.getCycleStatusCount(periodTaskList, Const.TASK_STOP));
        sb.append("\n\n\n");

        List<Map> currentDayperiodTaskMap = this.periodTaskService.getStatusCount(warnConf.getAppCode(), bizDate);
        sb.append("今日业务的周期任务执行情况:").append(Const.MAIL_LINE_FEED);
        sb.append("   待执行任务数量:").append(RamHelper.getCycleStatusCount(currentDayperiodTaskMap, Const.TASK_TODO_RUN));
        sb.append(",正在执行任务数量:").append(RamHelper.getCycleStatusCount(currentDayperiodTaskMap, Const.TASK_RUNNING)).append(Const.MAIL_LINE_FEED);
        sb.append("   执行成功任务数量:").append(RamHelper.getCycleStatusCount(currentDayperiodTaskMap, Const.TASK_SUCCESS));
        sb.append(",执行失败任务数量:").append(RamHelper.getCycleStatusCount(currentDayperiodTaskMap, Const.TASK_FAIL));
        sb.append("   置虚任务数量:").append(RamHelper.getCycleStatusCount(currentDayperiodTaskMap, Const.TASK_HAND_SUCCESS));
        sb.append("   暂停中任务数量:").append(RamHelper.getCycleStatusCount(currentDayperiodTaskMap, Const.TASK_STOP));
        sb.append("\n\n\n");

        List<Map> cronTaskList = this.cronTaskService.countGroupByExecStatus(warnConf.getAppCode());
        sb.append("总的定时任务执行情况:").append(Const.MAIL_LINE_FEED);
        sb.append("   正在执行任务数量:").append(RamHelper.getCronStatusCount(cronTaskList, Const.CRON_TASK_RUNNING));
        sb.append(",执行成功任务数量:").append(RamHelper.getCronStatusCount(cronTaskList, Const.CRON_TASK_SUCCESS)).append(Const.MAIL_LINE_FEED);
        sb.append("   执行失败任务数量:").append(RamHelper.getCronStatusCount(cronTaskList, Const.CRON_TASK_FAIL));
        sb.append("\n\n\n");

        List<Map> currentDaycronTaskMap = this.cronTaskService.countGroupByExecStatus(warnConf.getAppCode(), bizDate);
        sb.append("今日业务的定时任务执行情况:").append(Const.MAIL_LINE_FEED);
        sb.append("   正在执行任务数量:").append(RamHelper.getCronStatusCount(currentDaycronTaskMap, Const.CRON_TASK_RUNNING));
        sb.append(",执行成功任务数量:").append(RamHelper.getCronStatusCount(currentDaycronTaskMap, Const.CRON_TASK_SUCCESS)).append(Const.MAIL_LINE_FEED);
        sb.append("   执行失败任务数量:").append(RamHelper.getCronStatusCount(currentDaycronTaskMap, Const.CRON_TASK_FAIL));
        return sb.toString();
    }

    /**
     * 指定任务指定时间内未完成
     */
    public void appointTimeTaskNoFinish(String nowTime) {
        log.debug("指定任务指定时间内未完成【ScheduledService.appointTimeTaskNoFinish-开始】");
        try {
            String nowDate = HdrsDateUtil.format(new Date(), "yyyyMMdd");
            List<WarnConf> warnConfList = this.warnConfService.selectWarnConfList(Const.TASK_APPOINT_TIME_NO_FINISH, false);
            for (WarnConf warnConf : warnConfList) {
                if (appManager.getValidAppSession(warnConf.getAppCode()) != null) {
                    if (nowTime.equals(warnConf.getParam1()) && "2".equals(warnConf.getIsRegionAll())) {
                        this.appointTimeCycleTaskNoFinish(warnConf, nowDate, nowTime);
                    }
                    if (nowTime.equals(warnConf.getParam1()) && "3".equals(warnConf.getIsRegionAll())) {
                        this.appointTimeCronTaskNoFinish(warnConf, nowDate, nowTime);
                    }
                }
            }
        } catch (Exception e) {
            log.error("指定任务指定时间内未完成异常：{}", e.getMessage());
        }
        log.debug("指定任务指定时间内未完成【ScheduledService.appointTimeTaskNoFinish-结束】");
    }

    private void appointTimeCycleTaskNoFinish(WarnConf warnConf, String nowDate, String nowTime) {
        if (StringUtils.isNotBlank(warnConf.getRegionJson())) {
            String[] taskCodes = warnConf.getRegionJson().split(",");
            if (taskCodes != null && taskCodes.length > 0) {
                Map noticInf = this.warnConfService.getNoticInf(warnConf);
                String appName = appManager.getValidAppSession(warnConf.getAppCode()).getAppName();
                String subject = appName + "指定周期任务" + nowTime + "前未完成的任务汇总";
                List<String> taskCodeList = Arrays.asList(taskCodes);
                List<PeriodTask> periodTaskList = this.periodTaskService.qryTaskByTaskCodes(warnConf.getAppCode(), nowDate, taskCodeList);
                if (CollectionUtils.isNotEmpty(periodTaskList)) {
                    List<String> jobCodeList = periodTaskList.stream().map(PeriodTask::getJobCode).collect(Collectors.toList());
                    String mailContent = this.getCycleTaskNoFinishContent(nowTime, periodTaskList, taskCodeList);
                    warnSendMethodService.add(noticInf, subject, mailContent, StringUtils.join(jobCodeList, ","), warnConf.getPkId());
                }
            }
        }
    }

    private String getCycleTaskNoFinishContent(String
                                                       nowTime, List<PeriodTask> periodTaskList, List<String> taskCodeList) {
        StringBuffer sb = new StringBuffer();
        sb.append("尊敬的管理员，您好！").append(Const.MAIL_LINE_FEED_DOUBLE);
        sb.append("    截止").append(nowTime).append("指定的周期任务未完成情况如下:").append(Const.MAIL_LINE_FEED);
        Integer count = taskCodeList.size();
        Map<String, String> cycleTaskMap = periodTaskList.stream().collect(Collectors.toMap(PeriodTask::getTaskCode, PeriodTask::getTaskStatus));
        for (String taskCode : taskCodeList) {
            if (!cycleTaskMap.containsKey(taskCode)) {
                sb.append("    任务编码为:" + taskCode).append("的任务还未生成实例").append(Const.MAIL_LINE_FEED);
            } else if (Const.TASK_TODO_RUN.equals(cycleTaskMap.get(taskCode))) {
                sb.append("    任务编码为:" + taskCode).append("的任务还未执行").append(Const.MAIL_LINE_FEED);
            } else if (Const.TASK_RUNNING.equals(cycleTaskMap.get(taskCode))) {
                sb.append("    任务编码为:" + taskCode).append("的任务正在执行").append(Const.MAIL_LINE_FEED);
            } else if (Const.TASK_FAIL.equals(cycleTaskMap.get(taskCode))) {
                sb.append("    任务编码为:" + taskCode).append("的任务执行失败").append(Const.MAIL_LINE_FEED);
            } else {
                count--;
            }
        }
        if (count == 0) {
            sb.append("指定的任务已全部执行完成");
        }
        return sb.toString();
    }

    private void appointTimeCronTaskNoFinish(WarnConf warnConf, String nowDate, String nowTime) {
        if (StringUtils.isNotBlank(warnConf.getRegionJson())) {
            String[] taskCodes = warnConf.getRegionJson().split(",");
            if (taskCodes != null && taskCodes.length > 0) {
                Map noticInf = this.warnConfService.getNoticInf(warnConf);
                String appName = appManager.getValidAppSession(warnConf.getAppCode()).getAppName();
                String subject = appName + "指定定时任务" + nowTime + "前未完成的任务汇总";
                List<String> taskCodeList = Arrays.asList(taskCodes);
                List<CronTask> cronTaskList = this.cronTaskService.qryTaskList(warnConf.getAppCode(), nowDate, taskCodeList);
                if (CollectionUtils.isNotEmpty(cronTaskList)) {
                    String mailContent = this.getCronTaskNoFinishContent(nowTime, cronTaskList, taskCodeList);
                    this.warnSendMethodService.add(noticInf, subject, mailContent, StringUtils.join(taskCodeList, ","), warnConf.getPkId());
                }
            }
        }
    }

    private String getCronTaskNoFinishContent(String nowTime, List<CronTask> cronTaskList, List<String> taskCodeList) {
        StringBuffer sb = new StringBuffer();
        sb.append("尊敬的管理员，您好！").append(Const.MAIL_LINE_FEED);
        sb.append("    截止").append(nowTime).append("指定的定时任务未完成情况如下:").append(Const.MAIL_LINE_FEED);
        StringBuffer sb2 = getCronTaskNoFinishContent(cronTaskList, taskCodeList);
        if ((sb2.length() == 0)) {
            sb.append("指定的任务已全部执行完成");
        } else {
            sb.append(sb2);
        }
        return sb.toString();
    }

    private StringBuffer getCronTaskNoFinishContent(List<CronTask> cronTaskList, List<String> taskCodeList) {
        StringBuffer sb2 = new StringBuffer();
        Set<String> taskCodeSet = new HashSet<>();
        for (CronTask cronTask : cronTaskList) {
            taskCodeSet.add(cronTask.getTaskCode());
            if (Const.CRON_TASK_RUNNING.equals(cronTask.getExecStatus())) {
                sb2.append("任务编码为:" + cronTask.getTaskCode()).append("的任务正在执行").append(Const.MAIL_LINE_FEED);
            } else if (Const.CRON_TASK_FAIL.equals(cronTask.getExecStatus())) {
                sb2.append("任务编码为:" + cronTask.getTaskCode()).append("的任务执行失败").append(Const.MAIL_LINE_FEED);
            }
        }
        if (taskCodeList.size() != taskCodeSet.size()) {
            for (String taskCode : taskCodeList) {
                if (!taskCodeSet.contains(taskCode)) {
                    sb2.append("任务编码为:" + taskCode).append("的任务未生成实例").append(Const.MAIL_LINE_FEED);
                }
            }
        }
        return sb2;
    }

    /**
     * 任务失败数量预警
     */
    public void taskFailOut(String nowTime) {
        log.debug("任务失败数量预警【ScheduledService.taskFailOut-开始】");
        try {
            List<WarnConf> warnConfList = this.warnConfService.selectWarnConfList(Const.TASK_FAIL_OUT, false);
            for (WarnConf warnConf : warnConfList) {
                if (nowTime.equals(warnConf.getParam1()) && appManager.getValidAppSession(warnConf.getAppCode()) != null) {
                    String appName = appManager.getValidAppSession(warnConf.getAppCode()).getAppName();
                    Integer count = this.periodTaskService.countTask(warnConf.getAppCode(), HdrsDateUtil.format(new Date(), "yyyyMMdd"), Const.TASK_FAIL);
                    if (count > Integer.valueOf(warnConf.getParam2())) {
                        StringBuffer sb = new StringBuffer();
                        sb.append("尊敬的管理员，您好！").append("\r\n");
                        sb.append("   今日周期任务失败数:" + count + "  超过了预警阀值: " + warnConf.getParam2());
                        this.doAddWarnLog(Const.TYPE_APP, warnConf.getAppCode(), appName,
                                Const.WARN_LEVEL_WARN, "周期任务" + Const.TASK_FAIL_OUT_NAME, sb.toString(), warnConf);
                    }
                    Integer num = this.cronTaskService.countTask(warnConf.getAppCode(), HdrsDateUtil.format(new Date(), "yyyy-MM-dd"), Const.CRON_TASK_FAIL);
                    if (num > Integer.valueOf(warnConf.getParam3())) {
                        StringBuffer sb = new StringBuffer();
                        sb.append("尊敬的管理员，您好！").append("\r\n");
                        sb.append("   今日定时任务失败数:" + num + "  超过了预警阀值: " + warnConf.getParam3());
                        this.doAddWarnLog(Const.TYPE_APP, warnConf.getAppCode(), appName,
                                Const.WARN_LEVEL_WARN, "定时任务" + Const.TASK_FAIL_OUT_NAME, sb.toString(), warnConf);
                    }
                }
            }
        } catch (Exception e) {
            log.error("任务失败数量预警异常：{}", e.getMessage());
        }
        log.debug("任务失败数量预警【ScheduledService.taskFailOut-结束】");
    }

    /**
     * 任务执行超时预警
     */
    public void taskRunningTimeOut() {
        log.debug("任务执行超时预警【ScheduledService.taskRunningTimeOut-开始】");
        try {
            List<WarnConf> warnConfList = this.warnConfService.selectWarnConfList(Const.TASK_RUNNING_TIME_OUT, false);
            for (WarnConf warnConf : warnConfList) {
                if (appManager.getValidAppSession(warnConf.getAppCode()) != null) {
                    if ("3".equals(warnConf.getIsRegionAll())) {
                        this.cronTaskRunningTimeOut(warnConf);
                    }
                }
            }
        } catch (Exception e) {
            log.error("任务执行超时预警异常：{}", e.getMessage());
        }
        log.debug("任务执行超时预警【ScheduledService.taskRunningTimeOut-结束】");
    }

    private void cycleTaskRunningTimeOut(PeriodTask periodTask) {
        log.warn("任务【" + periodTask.getTaskCode() + "】,业务日期【" + periodTask.getBizDate() + "】执行超时【killAgentTask】");
        List<WarnConf> warnConfList = warnConfService.getAppointTaskWarnList(periodTask.getAppCode(),
                Const.TASK_RUNNING_TIME_OUT, periodTask.getTaskCode(), Const.IS_CYCLE_TASK);
        for (WarnConf warnConf : warnConfList) {
            Map noticeMap = this.warnConfService.getNoticInf(warnConf);
            String subject = "周期任务【" + periodTask.getTaskName() + "】执行超时";
            String content = "尊敬的管理员，您好! 此任务执行超时，任务名称:  " + periodTask.getTaskName() + "   业务日期:  " + periodTask.getBizDate()
                    + "   执行开始时间:  " + periodTask.getExecStTs();
            this.warnSendMethodService.add(noticeMap, subject, content, periodTask.getJobCode(), warnConf.getPkId());
        }
    }

    /**
     * 定时任务实例表中没有超时时长，定义表中存在
     *
     * @param warnConf
     */
    private void cronTaskRunningTimeOut(WarnConf warnConf) {
        if (StringUtils.isNotBlank(warnConf.getRegionJson())) {
            String[] taskCodes = warnConf.getRegionJson().split(",");
            if (taskCodes != null && taskCodes.length > 0) {
                List<CronTask> cronTaskList = this.cronTaskService.qryTaskListByTaskStatus(warnConf.getAppCode(), Const.CRON_TASK_RUNNING, Arrays.asList(taskCodes));
                for (CronTask cronTask : cronTaskList) {
                    CronTaskDef cronTaskDef = cronTaskDefService.queryByTaskCode(cronTask.getTaskCode());
                    if (StringUtils.isNotBlank(cronTask.getExecStTs()) && StringUtils.isNotBlank(cronTaskDef.getOverPlanDuration())) {
                        if (isTimeOut(cronTask.getExecStTs(), cronTaskDef.getOverPlanDuration(), cronTaskDef.getPlanExecDuration())) {
                            String content = "尊敬的管理员，您好! 此任务执行超时，任务名称:  " + cronTask.getTaskName() + "   执行开始时间:  " + cronTask.getExecStTs();
                            this.doAddWarnLog(Const.TYPE_APP, warnConf.getAppCode(), appManager.getValidAppSession(warnConf.getAppCode()).getAppName(),
                                    Const.WARN_LEVEL_WARN, "定时任务【" + cronTask.getTaskName() + "】执行超时", content, warnConf);
                        }
                    }
                }
            }
        }
    }

    private boolean isTimeOut(String execStTs, String overPlanDuration, String planExecDuration) {
        Integer timeOutSecond = Integer.valueOf(StrUtil.blankToDefault(overPlanDuration, "0"))
                + Integer.valueOf(StrUtil.blankToDefault(planExecDuration, "0"));
        Date timeOut = cn.hutool.core.date.DateUtil.parseDateTime(execStTs).offset(DateField.SECOND, timeOutSecond);
        return HdrsDateUtil.format(new Date(), DatePattern.NORM_DATETIME_MINUTE_PATTERN)
                .equals(HdrsDateUtil.format(timeOut, DatePattern.NORM_DATETIME_MINUTE_PATTERN));
    }

    /**
     * 任务指定时间未完成数量预警
     */
    public void taskNoFinishOut(String nowTime) {
        log.debug("任务指定时间未完成数量预警【ScheduledService.taskNoFinishOut-开始】");
        try {
            List<WarnConf> warnConfList = this.warnConfService.selectWarnConfList(Const.TASK_NO_FINISH_OUT, false);
            for (WarnConf warnConf : warnConfList) {
                if (nowTime.equals(warnConf.getParam1())
                        && appManager.getValidAppSession(warnConf.getAppCode()) != null) {
                    Map noticeMap = this.warnConfService.getNoticInf(warnConf);
                    Date nowDate = new Date();
                    Integer noFinishCount = this.periodTaskService.countTask(warnConf.getAppCode(), Const.TASK_TODO_RUN);
                    if (noFinishCount >= Integer.valueOf(warnConf.getParam2())) {
                        String appName = appManager.getValidAppSession(warnConf.getAppCode()).getAppName();
                        String subject = appName + "定时统计任务未完成数量预警";
                        StringBuffer sb = new StringBuffer();
                        sb.append("尊敬的管理员，您好！").append(Const.MAIL_LINE_FEED);
                        String content = sb.toString() + appName + "中的任务，截止" + HdrsDateUtil.format(nowDate, HdrsDateUtil.YYYY_MM_DDHHMISS) + "有" + noFinishCount + "个任务未完成！";
                        this.warnSendMethodService.add(noticeMap, subject, content, null, warnConf.getPkId());
                    }
                }
            }
        } catch (Exception e) {
            log.error("任务指定时间未完成数量预警异常：{}", e.getMessage());
        }
        log.debug("任务指定时间未完成数量预警【ScheduledService.taskNoFinishOut-结束】");
    }

    /**
     * 失败任务超时未处理预警
     */
    public void failTaskHandleTimeOut(String nowTime) {
        log.debug("失败任务超时未处理预警【ScheduledService.failTaskNoHandleTimeOut-开始】");
        try {
            List<WarnConf> warnConfList = this.warnConfService.selectWarnConfList(Const.FAIL_TASK_NO_HANDLE_TIME_OUT, false);
            for (WarnConf warnConf : warnConfList) {
                if (nowTime.equals(warnConf.getParam1()) && appManager.getValidAppSession(warnConf.getAppCode()) != null) {
                    String appName = appManager.getValidAppSession(warnConf.getAppCode()).getAppName();
                    List<PeriodTask> periodTaskList = this.getHandleTimeOutFailCycleTask(warnConf);
                    List<CronTask> cronTaskList = this.getHandleTimeOutFailCronTask(warnConf);
                    //this.failTaskHandleTimeOutAddLog(appName, warnConf, periodTaskList, cronTaskList);
                    this.failTaskHandleTimeOutSendMail(appName, warnConf, periodTaskList, cronTaskList);
                }
            }
        } catch (Exception e) {
            log.error("失败任务超时未处理预警异常：{}", e.getMessage());
        }
        log.debug("失败任务超时未处理预警【ScheduledService.failTaskNoHandleTimeOut-结束】");
    }

    private void failTaskHandleTimeOutAddLog(String appName, WarnConf
            warnConf, List<PeriodTask> periodTaskList, List<CronTask> cronTaskList) {
        Map noticInf = this.warnConfService.getNoticInf(warnConf);
        for (PeriodTask periodTask : periodTaskList) {
            String title = "周期任务:" + periodTask.getTaskName() + " 执行失败超时未处理";
            String content = "任务名称:" + periodTask.getTaskName() + " 业务日期:" + periodTask.getBizDate();
            this.warnSendMethodService.add(noticInf, title, content, periodTask.getJobCode(), warnConf.getPkId());
        }
        for (CronTask cronTask : cronTaskList) {
            String title = "定时任务:" + cronTask.getTaskName() + " 执行失败超时未处理";
            String content = "任务名称:" + cronTask.getTaskName() + " 结束时间:" + cronTask.getExecEndTs();
            this.warnSendMethodService.add(noticInf, title, content, cronTask.getTaskCode(), warnConf.getPkId());
        }
    }

    private void failTaskHandleTimeOutSendMail(String appName, WarnConf
            warnConf, List<PeriodTask> periodTaskList,
                                               List<CronTask> cronTaskList) {
        Map noticInf = this.warnConfService.getNoticInf(warnConf);
        String subject = appName + "任务执行失败超时未处理汇总";
        String mailContent = this.getFailTaskHandleTimeOutMailContent(appName, warnConf, periodTaskList, cronTaskList);
        List<String> jobCodeList = new ArrayList<>();
        for (PeriodTask task : periodTaskList) {
            jobCodeList.add(task.getJobCode());
        }

        this.warnSendMethodService.add(noticInf, subject, mailContent, StringUtils.join(jobCodeList, ","), warnConf.getPkId());
    }

    private String getFailTaskHandleTimeOutMailContent(String appName, WarnConf
            warnConf, List<PeriodTask> periodTaskList, List<CronTask> cronTaskList) {
        StringBuffer sb = new StringBuffer();
        sb.append("尊敬的管理员，您好！").append(Const.MAIL_LINE_FEED);
        sb.append("   周期任务执行失败超时未处理汇总如下:").append(Const.MAIL_LINE_FEED);
        if (CollectionUtils.isNotEmpty(periodTaskList)) {
            for (PeriodTask periodTask : periodTaskList) {
                sb.append("   任务名称:" + periodTask.getTaskName() + ", 业务日期:" + periodTask.getBizDate()).append(Const.MAIL_LINE_FEED);
            }
        } else {
            sb.append("   没有超时未处理的失败任务！");
        }
        sb.append("   定时任务执行失败超时未处理汇总如下:").append(Const.MAIL_LINE_FEED);
        if (CollectionUtils.isNotEmpty(cronTaskList)) {
            for (CronTask cronTask : cronTaskList) {
                sb.append("   任务名称:" + cronTask.getTaskName() + ", 执行时间:" + cronTask.getExecStTs()).append(Const.MAIL_LINE_FEED);
            }
        } else {
            sb.append("   没有超时未处理的失败任务！");
        }
        return sb.toString();
    }

    private List<CronTask> getHandleTimeOutFailCronTask(WarnConf warnConf) {
        String timeOut = HdrsDateUtil.getAheadTime("1", Integer.valueOf(warnConf.getParam2()), HdrsDateUtil.getCurrentTime());
        Map paramMap = new HashMap(16);
        paramMap.put("appCode", warnConf.getAppCode());
        paramMap.put("execStatus", Const.CRON_TASK_EXEC_STATUS_FAIL);
        paramMap.put("timeOut", timeOut);
//        paramMap.put("upTimeStr", DateUtil.format(DateUtil.addDay(-30), DateUtil.YYYY_MM_DDHHMISS));
        return this.cronTaskService.qryTaskList(paramMap);
    }

    private List<PeriodTask> getHandleTimeOutFailCycleTask(WarnConf warnConf) {
        String timeOut = HdrsDateUtil.getAheadTime("1", Integer.valueOf(warnConf.getParam2()), HdrsDateUtil.getCurrentTime());
        Map paramMap = new HashMap(16);
        paramMap.put("appCode", warnConf.getAppCode());
        paramMap.put("taskStatus", Const.TASK_FAIL);
        paramMap.put("timeOut", timeOut);
//        paramMap.put("upTimeStr", DateUtil.format(DateUtil.addDay(-30), DateUtil.YYYY_MM_DDHHMISS));
        return this.periodTaskService.qryPeriodTaskList(paramMap);
    }

    public void noticLogToSendLog() {
//        List<RunningStateLog> runLog = this.runningStateLogService.getLastTimeLog()
    }

    /**
     * 监控Running中和agent中以及DB中任务存在状况
     * 进行相应的补偿措施
     */
    public void monitorRunningAndAgent() {
        try {
            //先查询中心Running中的任务
            List<PeriodTask> runTaskList = runningManager.getRunTaskList();
            if (CollectionUtils.isNotEmpty(runTaskList)) {
                for (int i = 0; i < runTaskList.size(); i++) {
                    PeriodTask periodTask = runTaskList.get(i);
                    //去查询Agent
                    if (periodTask != null && Integer.valueOf(periodTask.getOverPlanDuration()) != 0) {
                        if (HdrsDateUtil.strToLong(periodTask.getExecStTs(), HdrsDateUtil.YYYY_MM_DDHHMISS)
                                + Integer.valueOf(periodTask.getPlanExecDuration()) * 1000
                                + Integer.valueOf(periodTask.getOverPlanDuration()) * 1000
                                < HdrsDateUtil.getMillis(new Date())) {
                            AgentSession agentSession = this.agentManager.getAgentSession(periodTask.getAgentCode());
                            String msg = agentSession.qryAgentRunning(periodTask);
                            if (Const.NOT_EXIST.equals(msg)) {
                                //agent不存在此任务
                                this.runningExistAgent(periodTask, agentSession);
                            } else if (Const.EXIST.equals(msg)) {
                                //agent存在此任务
                                this.runningNoExistAgent(periodTask);
                            } else {
                                log.error("监控agent时查询agent出错----");
                                this.runningExistAgent(periodTask, agentSession);
                            }
                        }
                    }
                }
            }
        } catch (Exception e) {
            log.error("【monitorRunningAndAgent】异常: {}", e.getMessage());
        }
    }

    public void runningExistAgent(PeriodTask periodTask, AgentSession agentSession) {
        try {
            //查询DB
            String dbTaskStatus = periodTaskService.selectTaskStatus(periodTask);
            if (Const.TASK_RUNNING.equals(dbTaskStatus)) {
                log.info("【DB状态为2 修改状态为1 任务编码为】: {}", periodTask.getTaskCode());
                //DB状态为2 修改状态为1
                Map<String, String> map = new HashMap<>(16);
                map.put("taskCode", periodTask.getTaskCode());
                map.put("bizDate", periodTask.getBizDate());
                map.put("taskStatus", Const.TASK_TODO_RUN);
                map.put("upTs", HdrsDateUtil.getCurTs());
                periodTaskService.updateTask(map);
            }
            log.debug("【DB状态不为2 删除内存 任务编码为】: {}", periodTask.getTaskCode());
            // DB状态不为2 删除内存
            runningManager.removeJob(periodTask, "监控，内存删除");
            // 更新session 任务
            AppSession appSession = appManager.getValidAppSession(periodTask.getAppCode());
            appSession.delTaskSession(periodTask);
            agentSession.delTaskSession(periodTask);
        } catch (Exception e) {
            log.error("【runningExistAgent】异常: {}", e.getMessage());

        }
    }

    public void runningNoExistAgent(PeriodTask periodTask) {
        try {
            //先判断running中任务是否超时
            String calLastedTime = HdrsDateUtil.calLastedTime(periodTask.getUpTs());
            if (Integer.parseInt(calLastedTime) > Integer.parseInt(periodTask.getOverPlanDuration())) {
                log.info("【running中任务超时 任务编码为】: {}", periodTask.getTaskCode());
                //超时
                //根据配置进行相应的操作
                if (Const.KILL.equals(taskStep)) {
                    //kill task
                    agentManager.killTask(periodTask);
                } else if (Const.WARN.equals(taskStep)) {
                    //不处理 预警
                }
            }
        } catch (Exception e) {
            log.error("【runningNoExistAgent】异常:{}", e.getMessage());
        }
    }

    public void monitorSessionTask() {
        log.debug("【监控】更新appManager/agentManager【ScheduledService.monitorSessionTask-开始】");
        try {
            appManager.delOverTsTask();
            agentManager.delOverTsTask();
        } catch (Exception e) {
            log.error("【monitorSessionTask】异常:{}", e.getMessage());
        }
    }

    /**
     * @Method: updateExceptionJob
     * @Param: []
     * @Return: void
     * @Description： 更新异常的作业，任务完成，作业未更新成功的
     * @Author： jd.chen
     * @Date： 2021/7/28 16:14
     * @Version： V1.0
     */
    public void updateExceptionJob() {
        try {
            upJobInAllTaskFinished();
            upFailJobInNoTaskFail();
            upJobAllTaskHandSuccess();
        } catch (Exception e) {
            log.error("【updateExceptionJob】异常:{}", e.getMessage());
        }
    }

    /**
     * @Method upJobAllTaskHandSuccess <br>
     * @Param  <br>
     * @Return void <br>
     * @Description 更新任务全部置虚 对应作业状态为成功<br>
     * @Author jian.qiao<br>
     * @Date 2023/11/29 17:20 <br>
     * @Version V1.0<br>
     */
    @Transactional(propagation = Propagation.REQUIRED, rollbackFor = Exception.class)
    public void upJobAllTaskHandSuccess(){
        List<PeriodJob> periodJobList = this.periodJobService.qryAllTaskHandSuccessJobNotSuccess();
        if (!ListUtil.isEmpty(periodJobList)) {
            String upTs = HdrsDateUtil.getCurTs();
            for (PeriodJob job : periodJobList) {
                // 不存在未置虚的任务
                if (countUnAppointTask(job.getJobCode(), job.getBizDate(), Const.PERIOD_TASK_STATUS_ZX) == 0) {
                    Map map = new HashMap(16);
                    map.put("jobStatus", Const.JOB_DO_SUCCESS);
                    map.put("jobCode", job.getJobCode());
                    map.put("bizDate", job.getBizDate());
                    map.put("upTs", upTs);
                    this.periodJobService.updateJob(map);
                }
            }
        }
    }

    @Transactional(propagation = Propagation.REQUIRED, rollbackFor = Exception.class)
    public void upJobInAllTaskFinished() {
        List<PeriodJob> periodJobList = this.periodJobService.qryJobDoingButAllTaskFinished();
        if (!ListUtil.isEmpty(periodJobList)) {
            String upTs = HdrsDateUtil.getCurTs();
            for (PeriodJob job : periodJobList) {
                // 不存在未执行完成的任务
                if (countUnAppointTask(job.getJobCode(), job.getBizDate(), Const.PERIOD_TASK_STATUS_SUCCESS) == 0){
                    Map map = new HashMap(16);
                    map.put("jobStatus", Const.JOB_DO_SUCCESS);
                    map.put("jobCode", job.getJobCode());
                    map.put("bizDate", job.getBizDate());
                    map.put("upTs", upTs);
                    this.periodJobService.updateJob(map);
                }
            }
        }
    }

    @Transactional(isolation = Isolation.SERIALIZABLE)
    public void upFailJobInNoTaskFail() {
        List<PeriodJob> periodJobList = this.periodJobService.qryFailJobInNoTaskFail();
        if (!ListUtil.isEmpty(periodJobList)) {
            String upTs = HdrsDateUtil.getCurTs();
            for (PeriodJob job : periodJobList) {
                Map map = new HashMap(16);
                map.put("jobStatus", Const.JOB_TO_DO);
                map.put("jobCode", job.getJobCode());
                map.put("bizDate", job.getBizDate());
                map.put("upTs", upTs);
                this.periodJobService.updateJob(map);
            }
        }
    }

    /**
     * @Method countUnAppointTask <br>
     * @Param jobCode
     * @Param bizDate
     * @Param taskStatus <br>
     * @Return  <br>
     * @Description 查询是否存在未完成/未置虚的任务，注意 条件是task_status != xxx<br>
     * @Author Qiao jian<br>
     * @Date 2024/7/26 15:12 <br>
     * @Version V1.0<br>
     */
    public Integer countUnAppointTask(String jobCode, String bizDate,String taskStatus) {
        Map<String,String> paramMap = new HashMap<>(16);
        paramMap.put("taskStatus", taskStatus);
        paramMap.put("jobCode", jobCode);
        paramMap.put("bizDate", bizDate);
        int count = this.periodTaskService.countUnAppointTask(paramMap);
        log.info("补偿机制-更新作业成功前检查任务状态，taskStatus：{},数量：{}",taskStatus, count);
        return count;
    }
}
